import requests
import pandas as pd
from lxml import etree

import UserAgents

def get_today_data(day):
    base_url = 'http://www.100ppi.com/sf/day-{}.html'
    url = base_url.format(day)

    ua = UserAgents.ua_generator()

    response = requests.request('GET', url, headers={'User-agent': ua})
    content = response.text
    html = etree.HTML(content)
    table = html.xpath("//table[@id='fdata']//tr[@align='center'] | "
                       "//table[@id='fdata']//tr//td[@colspan='8']")

    data_list = []
    for e in table:
        r = e.xpath("./text() | "
                    ".//a/text() | "
                    ".//td/text() | "
                    ".//font/text()")
        raw = []
        for i in r:
            raw_data = i.strip("\r | \n | \t | '' | \xa0 ")
            if raw_data:
                raw.append(raw_data)
            else:
                continue

        data_list.append(raw)

    if len(data_list) <= 1:
        return None
    else:
        set_data_list = []
        for lista in data_list:
            if len(lista) == 1:
                global exchange
                exchange = lista[0]
            else:
                lista.append(day)
                lista.append(exchange)
                set_data_list.append(lista)

        today_data = set_data_list

        header = ["name", "spot", "nearcode", "nearp", "nearbasis", "nearbasisper", "main_code",
                  "main_price", "basis", "basispercent", "date", "exchange"]

        today_table = pd.DataFrame(today_data, columns=header)
        today_table.drop(today_table.columns[[2, 3, 4, 5, 9]], axis=1, inplace=True)
        today_table.set_index('name', drop=False, inplace=True)
    return today_table


def get_piece(name, data):
    dif = {'菜籽油': '菜籽油OI',
           '甲醇': '甲醇MA',
           '动力煤': '动力煤ZC',
           'PVC': '聚氯乙烯',
           '沥青': '石油沥青',
           '热卷': '热轧卷板',
           }
    if name in list(dif.keys()):
        name = dif[name]
    try:
        tr = data.loc[name]
    except:
        piece = pd.DataFrame(
            columns=["name", "spot", "main_code", "main_price", "basis", "date", "exchange"])
    else:
        if isinstance(tr, pd.DataFrame):
            piece = pd.DataFrame(tr)
        elif isinstance(tr, pd.Series):
            piece = pd.DataFrame(tr).T
    return piece


# ["name", "spot", "maincode", "mainprice", "basis", "basispercent", "date", "exchange"]
# t1 = get_today_data('2019-11-04')
# t2 = get_today_data('2019-11-05')
# t3 = get_today_data('2019-11-02')
# t = pd.concat([t1, t2])
# print(t)
# print(t3)
# x = get_piece('铜', t)
# y = get_piece('铜', t1)
# z = get_piece('铜', t3)
# print(x, x.shape)
# print(y, y.shape)
# print(z, z.shape)
# m = pd.concat([x, y, z])
# print(m)
#        spot maincode mainprice basis basispercent        date exchange
# 铜  46810.00     1912     46600   210        0.45%  2019-10-10  上海期货交易所


# HEADER = ["name", "spot", "nearcode", "nearp", "nearbasis", "nearbasisper", "maincode",
#           "mainprice", "basis", "basispercent", "date", "exchange"]
# table = pd.DataFrame(t, columns=HEADER)
# table.drop(table.columns[[2, 3, 4, 5]], axis=1, inplace=True)
# table.set_index(['name'], inplace=True)
#
#
# print(table)

'''
            spot maincode mainprice  basis basispercent        date exchange
name                                                                        
铜       46810.00     1912     46600    210        0.45%  2019-10-10  上海期货交易所
螺纹钢      3750.00     2001      3412    338        9.01%  2019-10-10  上海期货交易所
锌       19450.00     1911     18790    660        3.39%  2019-10-10  上海期货交易所
铝       14000.00     1912     13880    120        0.86%  2019-10-10  上海期货交易所
'''
# nd = ['铜', '螺纹钢', '锌', '铝', '黄金', '线材', '天然橡胶', '铅', '白银', '石油沥青', '热轧卷板', '镍', '锡', '纸浆',
#       '不锈钢', 'PTA', '白糖', '棉花', '普麦', '菜籽油OI', '玻璃', '菜籽粕', '油菜籽', '硅铁', '锰硅', '甲醇MA', '动力煤ZC',
#       '棉纱', '尿素', '棕榈油', '聚氯乙烯', '聚乙烯', '豆一', '豆粕', '豆油', '玉米', '焦炭', '焦煤', '铁矿石', '鸡蛋',
#       '聚丙烯', '玉米淀粉', '苯乙烯']
#
# my_watchlist_specs = ['PTA', '白糖', '棉花', '菜籽油', '玻璃', '菜籽粕', '硅铁', '锰硅', '甲醇', '动力煤', '棉纱', '尿素',
#                       '棕榈油', 'PVC', '聚乙烯', '豆一', '豆粕', '豆油', '玉米', '焦炭', '焦煤', '铁矿石', '鸡蛋', '聚丙烯', '玉米淀粉', '苯乙烯', '乙二醇',
#                       '铜', '螺纹钢', '锌', '铝', '黄金', '天然橡胶', '铅', '白银', '沥青', '热卷', '镍', '锡', '纸浆', '不锈钢', '燃料油',
#                       '沪原油', '20号胶']
#
# common = [x for x in nd if x in my_watchlist_specs]
# dif = [y for y in (nd+my_watchlist_specs) if y not in common]
#
# print(common, '\n', dif)
# common: ['铜', '螺纹钢', '锌', '铝', '黄金', '天然橡胶', '铅', '白银', '镍', '锡', '纸浆', '不锈钢', 'PTA', '白糖',
# '棉花', '玻璃', '菜籽粕', '硅铁', '锰硅', '棉纱', '尿素', '棕榈油', '聚乙烯', '豆一', '豆粕', '豆油', '玉米', '焦炭',
# '焦煤', '铁矿石', '鸡蛋', '聚丙烯', '玉米淀粉', '苯乙烯']

# dif: ['线材', '石油沥青', '热轧卷板', '普麦', '菜籽油OI', '油菜籽', '甲醇MA', '动力煤ZC', '聚氯乙烯', '菜籽油',
# '甲醇', '动力煤', 'PVC', '乙二醇', '沥青', '热卷', '燃料油', '沪原油', '20号胶']

# dif = {'菜籽油': '菜籽油OI',
#        '甲醇': '甲醇MA',
#        '动力煤': '动力煤ZC',
#        'PVC': '聚氯乙烯',
#        '沥青': '石油沥青',
#        '热卷': '热压卷板',
#        }